package com.github.starry.fw.social.web.spider.wechat.tasklet;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.stereotype.Component;

@Component
public class GetWikiTasklet implements Tasklet {

	private static final Logger logger = LoggerFactory.getLogger(GetWikiTasklet.class);
	
	private String wikiUrl = "https://mp.weixin.qq.com/wiki/home/";
	
	private String directory = "E:/Temp/20160921";
	
	public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
		
		Document doc = saveUrl2File(wikiUrl,FilenameUtils.concat(directory, "home.html"));
		//Document doc = Jsoup.parse(new File(FilenameUtils.concat(directory, "home.html")),"UTF-8");
		
		List<String> urlList = getUrlList(doc);
		int count = 0;
		DecimalFormat   df   =   new   DecimalFormat("00");
		for (String url : urlList) {
			saveUrl2File(wikiUrl+url,
					FilenameUtils.concat(directory, df.format(count++) + ".html"));
		}
		
		return RepeatStatus.FINISHED;
	}
	
	private List<String> getUrlList(Document homeDoc) {
		List<String> urlList = new ArrayList<String>();
		
		Iterator<Element> linkIterator = homeDoc.select(".portal li a[href]").iterator();

		while(linkIterator.hasNext()){
			Element link = linkIterator.next();
			String href = link.attr("href");
			if(StringUtils.startsWith(href, "http")) continue;
			urlList.add(href);
		}

		return urlList;
	}
	
	private Document saveUrl2File(String url, String fileName) throws IOException {
		logger.info(url);
		Document doc = Jsoup.connect(url).get();
		
		FileUtils.writeStringToFile(new File(fileName), doc.html(),StandardCharsets.UTF_8);
		
		return doc;
	}

}